#coding=utf-8
import httplib2,urllib2,time
from BeautifulSoup import BeautifulSoup
import re, htmlentitydefs

class NoWebPageError(Exception):

    def __init__(self, url):
        self.url = url
    def __str__(self):
        return "Cannot get the web page from %s."%self.url
         
def get_soup(url, is_cached=False, encoding=None, try_times=10, timeout=15):

    if is_cached:
        h = httplib2.Http("./stcache", timeout)
    else:
        h = httplib2.Http(timeout=timeout)
        
    for i in range(try_times):
        try:
            resp, content = h.request(url)
        except Exception, e:
            print e
        else:
            if resp.status != 200:
                print 'status:',resp.status
                continue
            
            if resp['content-type'] and not resp['content-type'].startswith('text'): 
                print 'content-type:',resp['content-type']
                continue
            
            if encoding:
                content = content.decode(encoding, 'ignore')
            s = BeautifulSoup(content)
            return s
    
    raise NoWebPageError(url)

def u2utf8(u):
    return u.encode('u8')

def unescape(text):

    '''Removes HTML or XML character references and entities from a text string.
    @param text The HTML (or XML) source text.
    @return The plain text, as a Unicode string, if necessary.
    '''
    def fixup(m):
        text = m.group(0)
        if text[:2] == "&#":
            # character reference
            try:
                if text[:3] == "&#x":
                    return unichr(int(text[3:-1], 16))
                else:
                    return unichr(int(text[2:-1]))
            except ValueError:
                pass
        else:
            # named entity
            try:
                text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
            except KeyError:
                pass
        return text # leave as is
    return re.sub("&#?\w+;", fixup, text)
    
        
def get_text(soup):
    '''get unescaped text from html source'''
    
    texts = [t.strip() for t in soup.findAll(text=True)]
    return unescape('\n'.join(texts))

def get_string(item):
    while True:
        r = item.string
        if r:
            r = unescape(r) 
            return r.strip()
        try:
            item = item.contents[0]
        except:
            return None
